﻿	// Copyright 2010 Edmundo Pavel Soriano Morales

    // This file is part of eltokenizer.

    // eltokenizer is free software: you can redistribute it and/or modify
    // it under the terms of the GNU General Public License as published by
    // the Free Software Foundation, either version 3 of the License, or
    // (at your option) any later version.

    // eltokenizer is distributed in the hope that it will be useful,
    // but WITHOUT ANY WARRANTY; without even the implied warranty of
    // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    // GNU General Public License for more details.

    // You should have received a copy of the GNU General Public License
    // along with eltokenizer.  If not, see <http://www.gnu.org/licenses/>.
	
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Text.RegularExpressions;
namespace Tokenizer
{
    class Documento
    {
        public List<Token> tiposEnDocumento { get; set; }
        //public List<Token> tiposDocumento { get; set; }
        public string texto { get; set; }
        public string pathDocumento { get; set; }
        public int numeroTokens { get; set; }

        public Documento(string texto)
        {
         //   this.pathDocumento = pathDocumento;
            this.texto = texto;
            this.tiposEnDocumento = Tokeniza(this.texto);
            this.numeroTokens = tiposEnDocumento.Count();
            SacaFrecuencias();
            ObtenTipos();

        }
        public List<Token> Tokeniza(string texto)
        {

            string palabras = @"(\b[^\s\d-]+\b)";
            string mail = @"(\b?[\w]+@([\w_]+?\.)*[a-zA-Z]{2,6}\b?)";
            string telefono = @"(([0-9]{2}-?){4})";
            string noNumeros = @"(\b[^0-9]*\b)";
            texto = texto.Replace("\n", " ");
            texto = texto.Replace("\r", " ");
            Regex re = new Regex(mail);
            MatchCollection matches = Regex.Matches(texto,  telefono + "|" + mail + "|" + palabras );
            //MatchCollection matches = Regex.Matches(texto, palabras + "|" + mail + "|");
            List<Token> listaTokens = new List<Token>();
            for (int i = 0; i < matches.Count; i++)
                listaTokens.Add(new Token(matches[i].Value));

            return listaTokens;

        }

        public void ObtenTipos()
        {
            var tokensDistintos = this.tiposEnDocumento.GroupBy(p => p.token).Select(p => p.First());
            this.tiposEnDocumento = tokensDistintos.ToList();
        }

        public void SacaFrecuencias()
        {


            foreach (var token in this.tiposEnDocumento)
            {
                token.freqAbsoluta = this.tiposEnDocumento.Count(t => t.token == token.token);
            }
            this.tiposEnDocumento.ForEach(l => l.freqRelativa = (double) l.freqAbsoluta / this.numeroTokens);



            //Dictionary<string, int> tokensCuenta = new Dictionary<string, int>();
            //var listaTokensOrdenada = this.tokensDocumento.GroupBy(t => t);
            //foreach (var palabra in listaTokensOrdenada)
            //{
            //    tokensCuenta.Add(palabra.Key, palabra.Count());

            //}

            
        }

    
    }//clase Documento

    class Token
    {
        public string token { get; set; }
        public double tfIdf { get; set; }
        public int freqAbsoluta { get; set; }
        public double freqRelativa { get; set; }
        public int numDocsAparece { get; set; } 
        public Token(string token)
        {
            this.token = token;
        }

    }



}
